#!/usr/bin/env  python
import json
import time
from datetime import datetime, timedelta
from urllib.parse import quote, urlencode
from uuid import uuid4

from html5_parser import parse
from lxml import etree
from mechanize import Request

from calibre import browser
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.date import local_tz
from calibre.web.feeds.news import BasicNewsRecipe


def safe_dict(data, *names):
    ans = data
    for x in names:
        ans = ans.get(x) or {}
    return ans


def process_web_list(li_node):
    li_html = ''
    for li in li_node['items']:
        if li.get('textHtml'):
            li_html += f'<li>{li["textHtml"]}</li>'
        elif li.get('textJson'):
            li_html += f'<li>{parse_textjson(li["textJson"])}</li>'
        else:
            li_html += f'<li>{li.get("text", "")}</li>'
    return '<ul>' + li_html + '</ul>'


def process_info_box(bx):
    info = ''
    for x in safe_dict(bx, 'components'):
        info += f'<blockquote>{process_web_node(x)}</blockquote>'
    return info


def parse_txt(ty):
    typ = ty.get('type', '')
    children = ty.get('children', [])
    href = '#'
    attributes = ty.get('attributes') or ()
    for a in attributes:
        if a.get('name') == 'href':
            href = a.get('value', href)
            break

    tag_map = {
        'text': lambda: [ty.get('value', '')],
        'scaps': lambda: [
            f'<span style="text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;">{"".join(parse_txt(c))}</span>'
            for c in children
        ],
        'bold': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children],
        'drop_caps': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children],
        'italic': lambda: [f'<i>{"".join(parse_txt(c))}</i>' for c in children],
        'linebreak': lambda: ['<br>'],
        'external_link': lambda: [
            f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children
        ],
        'internal_link': lambda: [
            f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children
        ],
        'ufinish': lambda: [text for c in children for text in parse_txt(c)],
        'subscript': lambda: [f'<sub>{"".join(parse_txt(c))}</sub>' for c in children],
        'superscript': lambda: [f'<sup>{"".join(parse_txt(c))}</sup>' for c in children],
    }

    if typ in tag_map:
        yield from tag_map[typ]()
    else:
        print('** ', typ)


def parse_textjson(nt):
    return ''.join(''.join(parse_txt(n)) for n in nt)


def process_web_node(node):
    ntype = node.get('type', '')
    if ntype == 'CROSSHEAD':
        if node.get('textHtml'):
            return f'<h4>{node.get("textHtml")}</h4>'
        return f'<h4>{node.get("text", "")}</h4>'
    elif ntype in ['PARAGRAPH', 'BOOK_INFO']:
        if node.get('textHtml'):
            return f'\n<p>{node.get("textHtml")}</p>'
        elif node.get('textJson'):
            return f'\n<p>{parse_textjson(node["textJson"])}</p>'
        return f'\n<p>{node.get("text", "")}</p>'
    elif (ntype == 'IMAGE') or (node.get('__typename', '') == 'ImageComponent'):
        alt = '' if node.get('altText') is None else node.get('altText')
        cap = ''
        if node.get('caption'):
            if node['caption'].get('textHtml') is not None:
                cap = node['caption']['textHtml']
            elif node['caption'].get('textJson') is not None:
                cap = parse_textjson(node['caption']['textJson'])
            elif node['caption'].get('text') is not None:
                cap = node['caption']['text']
        return f'<div><img src="{node["url"]}" title="{alt}"></div><div style="text-align:center; font-size:small;">{cap}</div>'
    elif ntype == 'PULL_QUOTE':
        if node.get('textHtml'):
            return f'<blockquote>{node.get("textHtml")}</blockquote>'
        elif node.get('textJson'):
            return f'<blockquote>{parse_textjson(node["textJson"])}</blockquote>'
        return f'<blockquote>{node.get("text", "")}</blockquote>'
    elif ntype == 'BLOCK_QUOTE':
        if node.get('textHtml'):
            return f'<blockquote><i>{node.get("textHtml")}</i></blockquote>'
        elif node.get('textJson'):
            return f'<blockquote><i>{parse_textjson(node["textJson"])}</i></blockquote>'
        return f'<blockquote><i>{node.get("text", "")}</i></blockquote>'
    elif ntype == 'DIVIDER':
        return '<hr>'
    elif ntype == 'INFOGRAPHIC':
        if node.get('fallback'):
            return process_web_node(node['fallback'])
    elif ntype == 'INFOBOX':
        return process_info_box(node)
    elif ntype == 'UNORDERED_LIST':
        if node.get('items'):
            return process_web_list(node)
    elif ntype:
        print('** ', ntype)
    return ''


def load_article_from_json(raw):
    # open('/t/raw.json', 'w').write(raw)
    body = ''
    try:
        data = json.loads(raw)['data']['findArticleByUrl']
    except Exception:
        try:
            data = json.loads(raw)['props']['pageProps']['cp2Content']
        except Exception:
            data = json.loads(raw)['props']['pageProps']['content']

    body += f'<div style="color: red; font-size:small; font-weight:bold;">{data.get("flyTitle", "")}</div>'
    body += f'<h1>{data["headline"]}</h1>'
    if data.get('rubric') and data.get('rubric') is not None:
        body += f'<div style="font-style: italic; color:#202020;">{data.get("rubric", "")}</div>'
    try:
        date = data['dateModified']
    except Exception:
        date = data['datePublished']
    dt = datetime.fromisoformat(date[:-1]) + timedelta(seconds=time.timezone)
    dt = dt.strftime('%b %d, %Y %I:%M %p')
    if data.get('dateline') is None:
        body += f'<p style="color: gray; font-size: small;">{dt}</p>'
    else:
        body += f'<p style="color: gray; font-size: small;">{dt + " | " + (data["dateline"])}</p>'
    main_image_url = safe_dict(data, 'leadComponent') or ''
    if main_image_url:
        body += process_web_node(data['leadComponent'])
    if data.get('byline'):
        if data['byline'] is not None:
            body += f'<p style="color: gray; font-size: small;"><i>{"By " + data["byline"]}</i></p>'
    for node in data.get('body'):
        body += process_web_node(node)
    return '<html><body><article>' + body + '</article></body></html>'


class NoArticles(Exception):
    pass


def get_content(url_):

    headers = {
        'User-Agent': 'TheEconomist-Liskov-android',
        'accept': 'multipart/mixed; deferSpec=20220824, application/json',
        'accept-encoding': 'gzip',
        'content-type': 'application/json',
        'x-app-trace-id': str(uuid4()),
        'x-economist-consumer': 'TheEconomist-Liskov-android',
        'x-teg-client-name': 'Economist-Android',
        'x-teg-client-os': 'Android',
        'x-teg-client-version': '4.40.0',
    }
    br = browser()
    req = Request(
        url_,
        headers=headers,
    )
    res = br.open(req)
    return res.read()


def process_url(url):
    if url.startswith('/'):
        url = 'https://www.economist.com' + url
    return url


class Economist(BasicNewsRecipe):
    title = 'The Economist'
    language = 'en_GB'
    encoding = 'utf-8'
    masthead_url = 'https://www.livemint.com/lm-img/dev/economist-logo-oneline.png'

    __author__ = 'Kovid Goyal, unkn0wn'
    description = (
        'Global news and current affairs from a European'
        ' perspective. Best downloaded on Friday mornings (GMT)'
    )
    extra_css = '''
        em { color:#202020; }
        img {display:block; margin:0 auto;}
    '''

    resolve_internal_links = True

    delay = 1
    browser_type = 'webengine'
    from_web = True
    recipe_specific_options = {
        'date': {
            'short': 'The date of the edition to download (YYYY-MM-DD format)',
            'long': 'For example, 2024-07-19',
        },
        'res': {
            'short': 'For hi-res images, select a resolution from the\nfollowing options: 834, 960, 1096, 1280, 1424',
            'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use from 480, 384, 360, 256.',
            'default': '600',
        },
        'de': {
            'short': 'Web Edition',
            'long': 'Yes/No. Digital Edition does not skip interactive articles.',
            'default': 'No',
        },
    }

    def get_browser(self, *args, **kwargs):
        kwargs['user_agent'] = (
            'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.103 Mobile Safari/537.36 Liskov'
        )
        br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
        return br

    def canonicalize_internal_url(self, url, is_link=True):
        if url.endswith('/print'):
            url = url.rpartition('/')[0]
        return BasicNewsRecipe.canonicalize_internal_url(self, url, is_link=is_link)

    def publication_date(self):
        edition_date = self.recipe_specific_options.get('date')
        if edition_date and isinstance(edition_date, str):
            return datetime.strptime(edition_date, '%Y-%m-%d').replace(tzinfo=local_tz)
        try:
            url = self.browser.open('https://www.economist.com/printedition').geturl()
            date_str = url.split('/')[-1]
            return datetime.strptime(date_str, '%Y-%m-%d').replace(tzinfo=local_tz)
        except Exception as e:
            self.log('Failed to fetch publication date with error: ' + str(e))
            return super().publication_date()

    def economist_return_index(self, ans):
        if not ans:
            raise NoArticles(
                'Could not find any articles, either the '
                'economist.com server is having trouble and you should '
                'try later or the website format has changed and the '
                'recipe needs to be updated.'
            )
        return ans

    def economist_test_article(self):
        self.from_web = False
        return [('Articles', [{'title': 'test',
            'url': 'https://www.economist.com/china/2025/09/10/china-is-ditching-the-dollar-fast'
        }])]

    def parse_index(self):
        # return self.economist_test_article()
        edition_date = self.recipe_specific_options.get('date')
        if edition_date and isinstance(edition_date, str):
            url = 'https://www.economist.com/weeklyedition/' + edition_date
            self.timefmt = ' [' + edition_date + ']'
        else:
            url = 'https://www.economist.com/weeklyedition'
        soup = self.index_to_soup(url)
        ans = self.economist_parse_index(soup)
        return self.economist_return_index(ans)

    def economist_parse_index(self, soup):
        script_tag = soup.find('script', id='__NEXT_DATA__')
        if script_tag is not None:
            data = json.loads(script_tag.string)
            # open('/t/raw.json', 'w').write(json.dumps(data, indent=2, sort_keys=True))
            self.description = safe_dict(
                data, 'props', 'pageProps', 'content', 'headline'
            )
            self.timefmt = (
                ' ['
                + safe_dict(data, 'props', 'pageProps', 'content', 'formattedIssueDate')
                + ']'
            )
            self.cover_url = (
                safe_dict(data, 'props', 'pageProps', 'content', 'cover', 'url')
                .replace(
                    'economist.com/',
                    'economist.com/cdn-cgi/image/width=960,quality=80,format=auto/',
                )
                .replace('SQ_', '').replace('_AS', '_AP')
            )
            self.log('Got cover:', self.cover_url)

            feeds = []

            for part in safe_dict(
                data, 'props', 'pageProps', 'content', 'headerSections'
            ) + safe_dict(data, 'props', 'pageProps', 'content', 'sections'):
                section = safe_dict(part, 'name') or ''
                if not section:
                    continue
                self.log(section)

                articles = []

                for ar in part['articles']:
                    title = safe_dict(ar, 'headline') or ''
                    url = process_url(safe_dict(ar, 'url') or '')
                    if not title or not url:
                        continue
                    desc = safe_dict(ar, 'rubric') or ''
                    sub = safe_dict(ar, 'flyTitle') or ''
                    if sub and section != sub:
                        desc = sub + ' :: ' + desc
                    self.log('\t', title, '\n\t', desc, '\n\t\t', url)
                    articles.append({'title': title, 'url': url, 'description': desc})
                feeds.append((section, articles))
            d = self.recipe_specific_options.get('de')
            if d and isinstance(d, str):
                if d.lower().strip() == 'yes':
                    self.from_web = True
                else:
                    self.from_web = False
            return feeds
        return []

    def preprocess_html(self, soup):
        width = '600'
        w = self.recipe_specific_options.get('res')
        if w and isinstance(w, str):
            width = w
        for img in soup.findAll('img', src=True):
            qua = (
                'economist.com/cdn-cgi/image/width=' + width + ',quality=80,format=auto/'
            )
            img['src'] = img['src'].replace('economist.com/', qua)
        return soup

    def preprocess_raw_html(self, raw, url):
        # open('/t/raw.html', 'wb').write(raw.encode('utf-8'))
        if self.from_web:
            if '/interactive/' in url:
                return ('<html><body><article><h1>Article not available.</h1><em>'
                        'This article is supposed to be read in a browser.'
                        '</em></article></body></html>')
            root_ = parse(raw)
            script = root_.xpath('//script[@id="__NEXT_DATA__"]')
            html = load_article_from_json(script[0].text)
        else:
            html = load_article_from_json(raw)
        root = parse(html)

        for x in root.xpath(
            '//*[name()="script" or name()="style" or name()="source" or name()="meta"]'
        ):
            x.getparent().remove(x)
        # the economist uses <small> for small caps with a custom font
        for init in root.xpath('//span[@data-caps="initial"]'):
            init.set('style', 'font-weight:bold;')
        for x in root.xpath('//small'):
            if x.text and len(x) == 0:
                x.text = x.text.upper()
                x.tag = 'span'
                x.set(
                    'style',
                    'text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;',
                )
        for h2 in root.xpath('//h2'):
            h2.tag = 'h4'
        for x in root.xpath('//figcaption'):
            x.set('style', 'text-align:center; font-size:small;')
        for x in root.xpath('//cite'):
            x.tag = 'blockquote'
            x.set('style', 'color:#404040;')
        raw = etree.tostring(root, encoding='unicode')
        return raw

    def get_article(self, url):
        query = {
            'operationName': 'ArticleDeeplinkQuery',
            'variables': '{{"ref":"{}"}}'.format(url),
            'query': 'query ArticleDeeplinkQuery($ref: String!, $includeRelatedArticles: Boolean = true ) { findArticleByUrl(url: $ref) { __typename ...ArticleDataFragment } }  fragment ContentIdentityFragment on ContentIdentity { articleType forceAppWebView leadMediaType }  fragment NarrationFragment on Narration { album bitrate duration filename id provider url isAiGenerated fileHash }  fragment ImageTeaserFragment on ImageComponent { altText height imageType source url width }  fragment PodcastAudioFragment on PodcastEpisode { id audio { url durationInSeconds } }  fragment ArticleTeaserFragment on Article { id tegId url rubric headline flyTitle brand byline dateFirstPublished dateline dateModified datePublished dateRevised estimatedReadTime wordCount printHeadline contentIdentity { __typename ...ContentIdentityFragment } section { tegId name } teaserImage { __typename type ...ImageTeaserFragment } leadComponent { __typename type ...ImageTeaserFragment } narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } podcast { __typename ...PodcastAudioFragment } }  fragment AnnotatedTextFragment on AnnotatedText { text textJson annotations { type length index attributes { name value } } }  fragment ImageComponentFragment on ImageComponent { altText caption { __typename ...AnnotatedTextFragment } credit height imageType mode source url width }  fragment BlockQuoteComponentFragment on BlockQuoteComponent { text textJson annotations { type length index attributes { name value } } }  fragment BookInfoComponentFragment on BookInfoComponent { text textJson annotations { type length index attributes { name value } } }  fragment ParagraphComponentFragment on ParagraphComponent { text textJson annotations { type length index attributes { name value } } }  fragment PullQuoteComponentFragment on PullQuoteComponent { text textJson annotations { type length index attributes { name value } } }  fragment CrossheadComponentFragment on CrossheadComponent { text }  fragment OrderedListComponentFragment on OrderedListComponent { items { __typename ...AnnotatedTextFragment } }  fragment UnorderedListComponentFragment on UnorderedListComponent { items { __typename ...AnnotatedTextFragment } }  fragment VideoComponentFragment on VideoComponent { url title thumbnailImage }  fragment InfoboxComponentFragment on InfoboxComponent { components { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...VideoComponentFragment } }  fragment InfographicComponentFragment on InfographicComponent { url title width fallback { __typename ...ImageComponentFragment } altText height width }  fragment ArticleDataFragment on Article { id url brand byline rubric headline layout { headerStyle } contentIdentity { __typename ...ContentIdentityFragment } dateline dateFirstPublished dateModified datePublished dateRevised estimatedReadTime narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } printFlyTitle printHeadline printRubric flyTitle wordCount section { tegId name articles(pagingInfo: { pagingType: OFFSET pageSize: 6 pageNumber: 1 } ) @include(if: $includeRelatedArticles) { edges { node { __typename ...ArticleTeaserFragment } } } } teaserImage { __typename type ...ImageComponentFragment } tegId leadComponent { __typename type ...ImageComponentFragment } body { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...InfoboxComponentFragment ...ImageComponentFragment ...VideoComponentFragment ...InfographicComponentFragment } footer { __typename type ...ParagraphComponentFragment } tags { name } ads { adData } podcast { __typename ...PodcastAudioFragment } }',  # noqa: E501
        }
        deep_url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(
            query, safe='()!', quote_via=quote
        )
        raw = get_content(deep_url)
        return raw

    def print_version(self, url):
        if self.from_web:
            return url
        art_cont = self.get_article(url)
        pt = PersistentTemporaryFile('.html')
        pt.write(art_cont)
        pt.close()
        return 'file:///' + pt.name
